{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is ‘last_expr’\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/data/home/marmot/camtrap/PyCharm/CameraTraps-benchmark')\n",
    "sys.path.append('/data/home/marmot/camtrap/PyCharm/CameraTraps-benchmark/detection/detector_eval')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import defaultdict\n",
    "\n",
    "import numpy as np\n",
    "from detection.detector_eval import detector_eval\n",
    "from data_management import cct_json_utils\n",
    "from visualization.visualization_utils import plot_precision_recall_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "detection_results_dir = '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904'\n",
    "results_path = [os.path.join(detection_results_dir, f) for f in os.listdir(detection_results_dir)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/5618_detections_train_cct_20_val_ss1_20190908093215.json',\n",
       " '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/5618_detections_train_cct_20_val_ss1_20190908093215_noprefix.json',\n",
       " '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/6704_detections_train_ss1_val_cct20_20190908092921.json',\n",
       " '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/8506_detections_train_cct_20_val_cct20_20190908093026.json',\n",
       " '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/948_detections_train_ss1_val_ss1_20190908092426.json']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_gt_classes = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ground truth: `[x, y, width, height]`\n",
    "    \n",
    "API output: `[x_min, y_min, width_of_box, height_of_box]`\n",
    "\n",
    "TF format: `[y_min, x_min, y_max, x_max]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_metrics(per_cat_metrics):\n",
    "    for cat, metrics in per_cat_metrics.items():\n",
    "        print('\\n' + str(cat))\n",
    "        print('Number of gt:', metrics['num_gt'])\n",
    "        print('Average precision', metrics['average_precision'])\n",
    "\n",
    "    mAP_from_cats = detector_eval.find_mAP(per_cat_metrics)\n",
    "    print('\\nmAP as the average of AP across the {} categories is {:.4f}'.format(num_gt_classes, mAP_from_cats))\n",
    "\n",
    "    precision_at_08_recall = detector_eval.find_precision_at_recall(per_cat_metrics[1]['precision'], per_cat_metrics[1]['recall'], \n",
    "                                                                    per_cat_metrics[1]['scores'],\n",
    "                                                                    recall_level=0.8)\n",
    "    print('Precision at 0.8 recall:', precision_at_08_recall)\n",
    "    \n",
    "    plot_precision_recall_curve(per_cat_metrics[1]['precision'], per_cat_metrics[1]['recall'], title='bbox level')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty_threshold = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on SS S1, val on SS S1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 151/2843 [00:00<00:01, 1508.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running per-object analysis...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2843/2843 [00:01<00:00, 1528.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing precision recall for each category...\n",
      "Number of ground truth in category 1 is 4371\n",
      "Number of ground truth in category 2 is 7\n",
      "\n",
      "one_class\n",
      "Number of gt: 4378\n",
      "Average precision 0.814277886637954\n",
      "\n",
      "1\n",
      "Number of gt: 4371\n",
      "Average precision 0.8155819235188658\n",
      "\n",
      "2\n",
      "Number of gt: 7\n",
      "Average precision 0.0\n",
      "\n",
      "mAP as the average of AP across the 2 categories is 0.4078\n",
      "Precision at 0.8 recall: (0.8925472179683512, 0.9179999828338623)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHCxJREFUeJzt3XuUHOV95vHvMzO6ge7oAkhCkkFctATLWOZyfMCwYBZxvCjO4WCIsY2XNTHLxV479pLYaxNin13sdbKJwXG0C74QAwb2xFYcEoKBoGAjgjDmJiCWxUUjAZLQbaQRMxrpt3+81TWl0WimR0xNT7eezzl9puvS1b/q7qmn662qtxURmJmZATTVugAzMxs+HApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKFjDkfSKpHP3M+0sSa1DUMPlkh4t+TnmSApJLWU+jx1cHApmZpZzKJiZWc6hYI3qfZJWStos6XuSRhcnSvpjSRuzpqaPFsZPkPRDSRskvSrpy5Kasml/Jenewrw3SXpQkvorRtLxkh6QtEnSS5IuzsafJukNSc2FeT8s6ZnsfpOk6yX9VtJbku6WNPmdvzxmvXMoWKP6KPAfgKOBY4EvF6YdDkwBZgCfAJZIOi6b9m1gAvAu4APAx4FPZtM+D5yUHS84A7gC+ET001eMpEOBB4A7gGnApcB3JP27iFgO7AD+feEhv5/NC3Ad8LtZLUcCm4Fbqn8ZzAbGoWCN6uaIWBMRm4CvkzbERf89Ijoi4hHg74GLs2/rHwH+KCLaIuIV4FvAxwAioh24DPgz4G+AayOimoPWHwJeiYjvRURXRPwK+H/ARdn0Oyv1SRoHXJCNA/gD4EsR0RoRHcANwEU+uGxl8QfLGtWawv1XSd+yKzZHxI5epk8BRmbDxWkzKgMR8a+SVpO+8d9dZS2zgVMlbSmMawFuz+7fAfxS0lXA7wG/iohXC4/9W0l7Co/dDUyv8rnNBsR7CtaoZhXuHwWsKwxPypp0ek7fCOwibYiL09ZWBiRdDYzK5v9ilbWsAR6JiImF29iIuAogIlaSwmcRezcdVR67qMdjR0fE2n2exWwQOBSsUV0taWZ2UPaPgR/3mP4nkkZmxwY+BNwTEbtJ3/6/LmmcpNnA50hNRUg6FvgaqQnpY8AXJS2oopafAcdK+pikEdntfZJOKMxzB+n4wZnAPYXx383qmZ3VMFXS4gG9EmYD4FCwRnUH8E/A6uz2tcK0N0gHbNcBPwI+HREvZtOuJR34XQ08mi3ntqwN/2+AmyLi6Yj4DSlsbpc0qq9CIqINOA+4JHvON4CbSHscFXcCZwEPRcTGwvi/AJYC/ySpDVgOnFr9y2A2MPKP7JiZWYX3FMzMLOdQMDOznEPBzMxyDgUzM8vV3cVrU6ZMiTlz5tS6DDOzuvLkk09ujIip/c1Xd6EwZ84cVqxYUesyzMzqiqRX+5/LzUdmZlbgUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs1xpoSDpNknrJT23n+mS9JeSVkl6RtLJZdViZmbVKXNP4fvA+X1MXwTMy25XAn9V7YLdsauZWTlKC4WIWAZs6mOWxcAPI1kOTJR0RH/L3b4dli2DN98crErNzKyillc0z2Dv39Ftzca93nNGSVeS9iaYOnUujz8OTU0wfYC/UrtrF3R2pr/btkFLCxx5ZP+Pi4A9e6CrC3bvTrc9e6C9HcaMgfHju8ft2bP3/eItAnbsSDXMmNE9ftu2tC6jRoE0sHU6EBHdNym9lj2nw9DUYmbDSy1DobdNTq8NQxGxBFgCcNxxC+OQQ/pvQmpvT3sVO3ak+6+/Dhs2pA12RPff006DmTPThroSGJVbezts2QJtbfD222lcd01pgw4pGLq6uocrKkFQmbdY86hRey+jqQlGj4apU6GjI93GjYNjjoGNG6G5OQXH/sKmeL+rK906O9Pfjo6916sYbBEwdmxafuVxxWCrvE6jR8PkySkAJ01KgbFnT3p9Jk2CkSP3DpvKa1S0ezds3QrTpqVA3r49/R0/fu/HFm89X7ft22HECJg4sXtcz/DavTutc2UeKa336NHpda7MP2JE35+h3vS2fsW/vd16m1YZV6nlkEP2vz7NzQOv0+xA1TIUWtn7x9VnsvePq/erowNefjn98x9+eBpevz7dXnihe+NX+Scr/jN2dqYN1C9/mTbQu3enDSKk+Xtu0CrjiuMrew67du39mMo/e+XW1JT+sVta0m3bthQykKbt3JmWJaVpAJs3p7D57W/TczQ1dX+j7y0Qi89f3KhUailugCrz7d6dNrITJvS+nGLQvPEGrFmTXquWlu5g3bMnBULP2orP2dyc5uvsTNOKG7mWlvTYnhvMvtaz8rii4uMry29u3nu9i5qaUk1jxqRQnDChOxQrt8rnpxigo0btHZY9/1Z7vKs4b889teK6VuaZOjWFcktLev4jj0yhNmZMChTv1dlgqWUoLAWukXQX6Tdnt0bEPk1H+7NrFzz/PDz7bPoGePrp8OKLaY+g+I85dizMmpU2XD1t3Zo2uh0d6R9z5Mj07XziRDj00P3/sw6VLVtSYLS0pL87dqQNwciR6W/xfktL2giWXXMlZCvP09mZAqyyMa7UUbnt3p32uCrTIA1HpPew8thKeFQCtHirjNuzJ71XbW377pUV3/NKUFdqlFIIjxjRXXtHR3rMli1771UUl9Uz6IuvbW9hU2yS6zlPsb6ej2tvT5/hospzd3Sk9379+vT6Vd7r557b+32YOzcF2Zgx6bWaMSM9ftKkNE9l3c36U9pvNEuq/BD5FOBN4KvACICI+K4kATeTzlBqBz4ZEf12f3rccQvj2mtXMG1a+mdZtw4OOyz9M1Q2FEcemb5VmTWSiPRFZv369AWho6M7YCdN6m5aq+y5NTfv3UQ2YkT3Xs/MmWmvZ+TItKfR3p7+b6ZNc3NVo5L0ZEQs7G++0vYUIuLSfqYHcPWBLr/Szl88yDtlSmpGMmtEUtqjKe7V9NTRkY5Bbd+emiIrJ1dIKQAqJza0taXxlea75ua099HcnPaWjzgifdnq7EzLHDMmBUZXV3r+AzkeY/Wh7n5PoWLPntTEM3162luYPTt9mM0OZqNGpaajanV1pS9Xa9fCpk0pLMaOTcfqRo3qPjBfaf4rNllNnZq+iEH6MjZtmsOiEdRtKLS3w4IF6QM8bVqtqzGrTy0t6X/ouOP2ndbW1t1EtXlz2uvo6Og+S2/zZli1KgXBqFEpOCZPTk1TRx+dvrRZ/am7UJDSB7OrK32Yzawc48Z1730fddS+0ytn8b36atrL2LIF3nornezx9NMpHBYsSMc7rH7UXSgAzJtX+zODzA52UtpDOPbY7nFvv50C4Y030rGLNWvS/+rxx6fjf3PnuolpuKvLUJgyJX0YzWx4GT0aTj013X/1VVi5Mh2kbmtL0x5/PDX3nnPOvtea2PDgt8XMSjF7drpB6qvstdfSSSGbN6fhU05Je/0+BXZ4cSiYWemmT+/uq+yxx1KXM8uWwYoVcOKJ6diDDQ8OBTMbUqefnk4pf/zxdIB6x4605/DBD/pY4XDgt8DMhlxTUwqHM85IF9u1tsIdd6SuanbvrnV1BzeHgpnVzOjRcO65qeuOdevgoYdSOFQ6jLSh51Aws5pqaYHzzktnJHV2pqake+6pdVUHL4eCmQ0bZ5yRLkrdvBl++tPUtGRDy6FgZsPKggXpwrfXXoOlS9MBaf8u+9Dx2UdmNqyMGAGLFqXTVp98MnVps24dfPjDta7s4OA9BTMblqZOTcca3nwzHYj+xS9qXdHBwaFgZsNWU1MKhg0b0umqVj6HgpkNay0tqZfWtjb40Y9qXU3jcyiY2bA3f34KhbY27zGUzaFgZsOeBGefnU5RfeaZWlfT2BwKZlYXKr8TvWFD+qEtK4dDwczqxuzZqQO9hx6qdSWNy6FgZnXjsMPSXsLmzbWupHE5FMysbkjpTKRNm9Ivu9ngcyiYWV2ZPTudhfTzn8POnbWupvE4FMysrowbl/pHeuut1Juq+0UaXA4FM6s706alcNiwAR55pNbVNBaHgpnVpZNPhvZ2H3QebA4FM6tLLS0weTJs317rShqLQ8HM6tb48em3F/y7zoPHoWBmdWvyZF/MNtgcCmZWt6ZMSRezbdtW60oah0PBzOpWSwsccUT6ER4bHKWGgqTzJb0kaZWk63uZfpSkhyU9JekZSReUWY+ZNZ7p09NZSPffX+tKGkNpoSCpGbgFWATMBy6VNL/HbF8G7o6I9wCXAN8pqx4za0zTp6crm9eurXUljaHMPYVTgFURsToiOoG7gMU95glgfHZ/ArCuxHrMrAE1N8PMmenUVHep/c6VGQozgDWF4dZsXNENwGWSWoH7gGt7W5CkKyWtkLRiy5YNZdRqZnVs8uQUCC+/XOtK6l+ZoaBexvXspeRS4PsRMRO4ALhd0j41RcSSiFgYEQsnTpxaQqlmVs8OPRS6utxz6mAoMxRagVmF4Zns2zx0BXA3QEQ8BowGppRYk5k1oLFjUyhs2lTrSupfmaHwBDBP0lxJI0kHkpf2mOc14BwASSeQQsHtQ2Y2IC0tcPjhKRTa22tdTX0rLRQiogu4BrgfeIF0ltHzkm6UdGE22+eBT0l6GrgTuDzCHeGa2cAdeWQ62Pzoo7WupL61lLnwiLiPdAC5OO4rhfsrgfeXWYOZHRymT4df/Qpee63WldQ3X9FsZg1BSqem7tjhs5DeCYeCmTWMo45KTUi/+U2tK6lfDgUzaxjjx6eDzmvXwp49ta6mPjkUzKxhNDWls5C2boUHHqh1NfXJoWBmDeWEE6CtzT+8c6BKPfvIzGyoNTfDmWemri9s4LynYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmuZZqZ5Q0A5hdfExELCujKDMzq42qQkHSTcBHgJXA7mx0AH2GgqTzgb8AmoH/GxH/s5d5LgZuyJb3dET8frXFm5nZ4Kp2T+F3geMioqPaBUtqBm4BPgi0Ak9IWhoRKwvzzAP+CHh/RGyWNK360s3MbLBVe0xhNTBigMs+BVgVEasjohO4C1jcY55PAbdExGaAiFg/wOcwM7NBVO2eQjvwa0kPAvneQkRc18djZgBrCsOtwKk95jkWQNIvSE1MN0TEP1ZZk5mZDbJqQ2FpdhsI9TIuenn+ecBZwEzgXySdGBFb9lqQdCVwJcD06UcNsAwzM6tWVaEQET+QNJLsmz3wUkTs6udhrcCswvBMYF0v8yzPlvWypJdIIfFEj+dfAiwBOP74hT2DxczMBklVxxQknQX8hnTg+DvAv0k6s5+HPQHMkzQ3C5RL2Hdv4yfA2dlzTCGFzuqqqzczs0FVbfPRt4DzIuIlAEnHAncC793fAyKiS9I1wP2k4wW3RcTzkm4EVkTE0mzaeZIqp7p+ISLeOvDVMTOzd6LaUBhRCQSAiPg3Sf2ejRQR9wH39Rj3lcL9AD6X3czMrMaqDYUVkm4Fbs+GPwo8WU5JZmZWK9WGwlXA1cB1pLOKlpGOLZiZWQOp9uyjDuDPspuZmTWoPkNB0t0RcbGkZ9n3GgMi4qTSKjMzsyHX357CZ7K/Hyq7EDMzq70+r1OIiNezuxuBNRHxKjAKeDf7XohmZmZ1rtoO8ZYBo7PfVHgQ+CTw/bKKMjOz2qg2FBQR7cDvAd+OiA8D88sry8zMaqHqUJB0Oun6hL/PxlX9q21mZlYfqg2Fz5J+DOdvs64q3gU8XF5ZZmZWC9Vep/AI8EhheDXpQjYzM2sg/V2n8L8j4rOS/o7er1O4sLTKzMxsyPW3p1Dp6+h/lV2ImZnVXp+hEBGVTu9WADsjYg+ApGbS9QpmZtZAqj3Q/CBwSGF4DPDzwS/HzMxqqdpQGB0R2ysD2f1D+pjfzMzqULWhsEPSyZUBSe8FdpZTkpmZ1Uq1F6B9FrhHUqW/oyOAj5RTkpmZ1Uq11yk8Iel44DjSj+y8GBG7Sq3MzMyGXFXNR5IOAf4b8JmIeBaYI8ndaZuZNZhqjyl8D+gETs+GW4GvlVKRmZnVTLWhcHREfAPYBRARO0nNSGZm1kCqDYVOSWPIurqQdDTQUVpVZmZWE9WeffRV4B+BWZJ+BLwfuLysoszMrDb6DQVJAl4k/cDOaaRmo89ExMaSazMzsyHWbyhEREj6SUS8l+4f2DEzswZU7TGF5ZLeV2olZmZWc9UeUzgb+LSkV4AdpCakiIiTyirMzMyGXrWhsKjUKszMbFjo75fXRgOfBo4BngVujYiuoSjMzMyGXn/HFH4ALCQFwiLgW6VXZGZmNdNf89H8iPgdAEm3Av9afklmZlYr/e0p5D2hutnIzKzx9RcK75a0Lbu1ASdV7kva1t/CJZ0v6SVJqyRd38d8F0kKSQsHugJmZjZ4+mw+iojmA12wpGbgFuCDpF5Vn5C0NCJW9phvHHAd8PiBPpeZmQ2Oai9eOxCnAKsiYnVEdAJ3AYt7me9PgW8Ab5dYi5mZVaHMUJgBrCkMt2bjcpLeA8yKiJ/1tSBJV0paIWnFli0bBr9SMzMDyg2F3n5vIfKJUhPw58Dn+1tQRCyJiIURsXDixKmDWKKZmRWVGQqtwKzC8ExgXWF4HHAi8M9Z9xmnAUt9sNnMrHbKDIUngHmS5koaCVwCLK1MjIitETElIuZExBxgOXBhRKwosSYzM+tDaaGQXddwDXA/8AJwd0Q8L+lGSReW9bxmZnbgqu0Q74BExH3AfT3GfWU/855VZi1mZta/MpuPzMyszjgUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHKlhoKk8yW9JGmVpOt7mf45SSslPSPpQUmzy6zHzMz6VlooSGoGbgEWAfOBSyXN7zHbU8DCiDgJuBf4Rln1mJlZ/8rcUzgFWBURqyOiE7gLWFycISIejoj2bHA5MLPEeszMrB9lhsIMYE1huDUbtz9XAP/Q2wRJV0paIWnFli0bBrFEMzMrKjMU1Mu46HVG6TJgIfDN3qZHxJKIWBgRCydOnDqIJZqZWVFLictuBWYVhmcC63rOJOlc4EvAByKio8R6zMysH2XuKTwBzJM0V9JI4BJgaXEGSe8B/hq4MCLWl1iLmZlVobRQiIgu4BrgfuAF4O6IeF7SjZIuzGb7JjAWuEfSryUt3c/izMxsCJTZfERE3Afc12PcVwr3zy3z+c3MbGB8RbOZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeVKDQVJ50t6SdIqSdf3Mn2UpB9n0x+XNKfMeszMrG+lhYKkZuAWYBEwH7hU0vwes10BbI6IY4A/B24qqx4zM+tfS4nLPgVYFRGrASTdBSwGVhbmWQzckN2/F7hZkiIi+lrw229D33OY2cFs505vIw5UmaEwA1hTGG4FTt3fPBHRJWkrcBiwsTiTpCuBK7OhznPOGb/64H3Ld02CEZtrXUXtHMzrfzCvOwxs/SXo6ITOXeXWNKSm0GPbOECzq5mpzFBQL+N6bsirmYeIWAIsAZC0ImLbwndeXn1K6/+21/8gdDCvO3j90/pH6etf5oHmVmBWYXgmsG5/80hqASYAm0qsyczM+lBmKDwBzJM0V9JI4BJgaY95lgKfyO5fBDzU3/EEMzMrT2nNR9kxgmuA+4Fm4LaIeF7SjcCKiFgK3ArcLmkVaQ/hkioWvaSsmuuE1//gdTCvO3j9h2T95S/mZmZW4Suazcws51AwM7PcsA2Fg72LjCrW/3OSVkp6RtKDkqo6B7ke9LfuhfkukhSSGuo0xWrWX9LF2fv/vKQ7hrrGMlXx2T9K0sOSnso+/xfUos4ySLpN0npJz+1nuiT9ZfbaPCPp5EEvIiKG3Y10YPq3wLuAkcDTwPwe8/wX4LvZ/UuAH9e67iFe/7OBQ7L7VzXK+lez7tl844BlwHJgYa3rHuL3fh7wFDApG55W67qHeP2XAFdl9+cDr9S67kFc/zOBk4Hn9jP9AuAfSNd4nQY8Ptg1DNc9hbyLjIjoBCpdZBQtBn6Q3b8XOEdSbxfD1aN+1z8iHo6I9mxwOek6kEZQzXsP8KfAN4C3h7K4IVDN+n8KuCUiNgNExPohrrFM1ax/AOOz+xPY9/qnuhURy+j7Wq3FwA8jWQ5MlHTEYNYwXEOhty4yZuxvnojoAipdZDSCata/6ArSt4dG0O+6S3oPMCsifjaUhQ2Rat77Y4FjJf1C0nJJ5w9ZdeWrZv1vAC6T1ArcB1w7NKUNCwPdNgxYmd1cvBOD1kVGnap63SRdBiwEPlBqRUOnz3WX1ETqUffyoSpoiFXz3reQmpDOIu0h/oukEyNiS8m1DYVq1v9S4PsR8S1Jp5OudToxIvaUX17Nlb7dG657Cgd7FxnVrD+SzgW+BFwYER1DVFvZ+lv3ccCJwD9LeoXUrrq0gQ42V/vZ/2lE7IqIl4GXSCHRCKpZ/yuAuwEi4jFgNKmzuINBVduGd2K4hsLB3kVGv+ufNaH8NSkQGqlNuc91j4itETElIuZExBzS8ZQLI2JFbcoddNV89n9COtEASVNIzUmrh7TK8lSz/q8B5wBIOoEUChuGtMraWQp8PDsL6TRga0S8PphPMCybj6K8LjLqQpXr/01gLHBPdnz9tYi4sGZFD5Iq171hVbn+9wPnSVoJ7Aa+EBFv1a7qwVPl+n8e+D+S/iup6eTyRvlCKOlOUrPglOyYyVeBEQAR8V3SMZQLgFVAO/DJQa+hQV5LMzMbBMO1+cjMzGrAoWBmZjmHgpmZ5RwKZmaWcyiYmVnOoWDWg6Tdkn4t6TlJfydp4iAv/3JJN2f3b5D0h4O5fLN3wqFgtq+dEbEgIk4kXQNzda0LMhsqDgWzvj1GocMxSV+Q9ETWl/2fFMZ/PBv3tKTbs3H/Mfutj6ck/VzS9BrUbzYgw/KKZrPhQFIzqTuFW7Ph80h9DJ1C6phsqaQzgbdIfVC9PyI2SpqcLeJR4LSICEn/Gfgi6Wpcs2HLoWC2rzGSfg3MAZ4EHsjGn5fdnsqGx5JC4t3AvRGxESAiKh0zzgR+nPV3PxJ4eUiqN3sH3Hxktq+dEbEAmE3amFeOKQj4H9nxhgURcUxE3JqN762/mG8DN0fE7wB/QOq4zWxYcyiY7UdEbAWuA/5Q0ghSJ23/SdJYAEkzJE0DHgQulnRYNr7SfDQBWJvd/wRmdcDNR2Z9iIinJD0NXBIRt2ddNT+W9Uy7Hbgs68Xz68AjknaTmpcuJ/1C2D2S1pK6+J5bi3UwGwj3kmpmZjk3H5mZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnl/j+bpGdNRL2jYwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "file_prefix = 'SER/'\n",
    "\n",
    "results_path = '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/948_detections_train_ss1_val_ss1_20190908092426.json'\n",
    "\n",
    "gt_db_path = '/beaver_disk/camtrap/ss_season1/benchmark/SnapshotSerengetiBboxesS01_20190903_val.json'\n",
    "\n",
    "detection_res = make_detection_res(results_path, file_prefix=file_prefix)\n",
    "\n",
    "gt_indexed = get_gt_db(gt_db_path)\n",
    "\n",
    "per_image_gts, per_image_detections = detector_eval.get_per_image_gts_and_detections(gt_indexed, detection_res)\n",
    "\n",
    "per_cat_metrics = detector_eval.compute_precision_recall_bbox(per_image_detections, per_image_gts, num_gt_classes, \n",
    "                                                         matching_iou_threshold=0.5)\n",
    "\n",
    "show_metrics(per_cat_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## empty vs non-empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9785689595575527\n"
     ]
    }
   ],
   "source": [
    "accuracy = detector_eval.compute_emptiness_accuracy(gt_indexed, detection_res, threshold=empty_threshold)\n",
    "\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on SS S1, val on CCT-20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 159/18202 [00:00<00:11, 1588.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running per-object analysis...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18202/18202 [00:11<00:00, 1596.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing precision recall for each category...\n",
      "Number of ground truth in category 1 is 18113\n",
      "Number of ground truth in category 2 is 0\n",
      "\n",
      "one_class\n",
      "Number of gt: 18113\n",
      "Average precision 0.713500230159196\n",
      "\n",
      "1\n",
      "Number of gt: 18113\n",
      "Average precision 0.7130783511904878\n",
      "\n",
      "2\n",
      "Number of gt: 0\n",
      "Average precision nan\n",
      "\n",
      "mAP as the average of AP across the 2 categories is 0.3565\n",
      "Precision at 0.8 recall: (0.47243504058944347, 1.0)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHtFJREFUeJzt3X2UHXWd5/H3J93pdEIeIAkRSDABCY4RkEALQZ6CIgLHJfPgcWBFxWVldH1cn5YZXYdx9OxR13F3FIdhDz7BgAQ8o1FQFERAJJJAEEIgTgxgOiEkgTw/d/d3//jVrb7pdLpvJ11dfbs/r3Pu6VtVv1v3W7e763Pr6VeKCMzMzABGlF2AmZkNHg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORRsyJH0vKQLDzBtrqTWAajhKkm/Kfg9ZkgKSY1Fvo8NLw4FMzPLORTMzCznULCh6o2SlknaKOk7kpqrJ0r6O0kbsl1N76oaP0HS9yWtl/SCpM9JGpFN+xdJd1a1/bKk+ySpt2Ik/ZmkX0p6RdJySe/Mxs+RtFZSQ1Xbv5D0ZPZ8hKRrJf1R0suS5kuaeOgfj1n3HAo2VL0LeBvwGuBE4HNV044CJgNTgfcCN0p6bTbtG8AE4HjgfOA9wPuyaZ8ETsmOF5wLXA28N3rpK0bSYcAvgVuBKcAVwLckvT4iFgLbgTdXveQ/Z20BPgr8eVbLMcBG4PraPwazvnEo2FD1zYhYFRGvAF8irYir/c+I2B0RDwB3Ae/Mvq3/NfC3EbE1Ip4Hvga8GyAidgBXAv8E3AJ8JCJqOWj9duD5iPhORLRFxOPAD4F3ZNNvq9QnaRxwaTYO4G+Az0ZEa0TsBq4D3uGDy1YU/2HZULWq6vkLpG/ZFRsjYns30ycDTdlw9bSplYGIeFTSStI3/vk11jIdOFPSpqpxjcDN2fNbgd9K+iDwl8DjEfFC1Wv/XVJH1WvbgVfV+N5mfeItBRuqjq16/mpgTdXwEdkuna7TNwB7SSvi6mmrKwOSPgSMytp/psZaVgEPRMThVY+xEfFBgIhYRgqfS9h311HltZd0eW1zRKze713M+oFDwYaqD0malh2U/Tvg9i7T/0FSU3Zs4O3AHRHRTvr2/yVJ4yRNBz5B2lWEpBOBL5J2Ib0b+IykU2uo5afAiZLeLWlk9nijpNdVtbmVdPzgPOCOqvE3ZPVMz2o4UtK8Pn0SZn3gULCh6lbgF8DK7PHFqmlrSQds1wD/BnwgIp7Npn2EdOB3JfCbbD7fzvbh3wJ8OSJ+HxH/QQqbmyWN6qmQiNgKXARcnr3nWuDLpC2OituAucCvImJD1fj/CywAfiFpK7AQOLP2j8Gsb+Sb7JiZWYW3FMzMLOdQMDOznEPBzMxyDgUzM8vV3cVrkydPjhkzZpRdhplZXXnsscc2RMSRvbWru1CYMWMGixcvLrsMM7O6IumF3lt595GZmVVxKJiZWc6hYGZmOYeCmZnlHApmZpZzKJiZWa6wUJD0bUnrJC09wHRJ+mdJKyQ9Kem0omoxM7PaFLml8F3g4h6mXwLMzB7XAP9SYC1mZlaDwkIhIh4EXumhyTzg+5EsBA6XdHRR9ZiZWe/KPKYwlX3vo9tK1b1wq0m6RtJiSYvXr18/IMWZmQ1HZYaCuhnX7R1/IuLGiGiJiJYjj+y16w4zMztIZYZCK/veXH0a+95c3czMBliZobAAeE92FtIcYHNEvFhiPWZmw15hvaRKqtyIfLKkVuDvgZEAEXEDcDdwKbAC2AG8r6hazMysNoWFQkRc0cv0AD5U1PubmVnf+YpmMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCxX2Cmpg0kELFkCu3fDzJkweXLZFVnR2tvTT6nzcTAioKMj/awM9/azo6PzNREwYkRnDY2NMGrUwdViNhCGbChs3w7r1sErr8CaNbB5cxr3hz/AMcfAiSfChg3pn7StLb1m1ChoaoItW+DZZ9O4446D2bNhzJiDqyMihdHu3XDYYbB3L2zcCBMmpJVERweMHJlWYm1taXjvXti5E9avT9PGjoWGhvRz9OjUrvLaykpn9+7UdvfuNG779s6Vz86dqU1zcxpXWUG1t6f5NWZ/BZWVWVtb+tzGjEntOzo6519Z6e3dm2qqXs7qFWNEWs5Ro1Ld7e3pNW1t6T1HZNuolVrWroWJE/d9j+qf1Y+2Nti6Nb2/BH/8Y5p/JQh6M3Jk+mIwblya39atsGNHej5yZBpub993mQ5WdaBUNDTARRfBlCmHNm+zIgy5UNi7F558Ep55Jg23taUV5IgRKRgk2LULXnop/eM3NKQ27e2dK8c9e9JrIK0gVq5Mrzv6aJg+PY3r6Oh8z4aG9KisnJctS+0PtEKpjD/Qt9dKQOzdm1ZSldc0NXWuTKvbdnR0vn9lpV5ZoVXqrMyno2PfFfKIEfvPs/p1leWoflQCqavK8lSv0Ku/JUP67Cu1dv0suo6vrrXrZ9nR0Vl/9bTt21P4VtpUz2P37vT+o0bBtm371tXd76oyrq2tM4Aqn0d3r+0pRCrL8/LLKZx//OMUjm96Exx11MF/6TDrb0MqFNatg0cfhU2b0rfUXbvSN/IJE9KWQUNDWhk8/HDaGti1K/1jTpyYVibNzWnlOXp02joA+O1v4cUX02s3bkwBsWdPCpDKCqKy4mlrSyvuyvgtW2D8+NR227Y0ffz49M29soLftSu9b2XlVlnRjRrV+S19xIj0nlu2pHlVr6AgvW7nzs5pI0fuO736m31Dw77fuCuBOHr0vrs7Kiu3ykodOlfc0Bmgve2WqSwTpG/mEyak39OoUSlca9HQkAK5UtvYsbW9brBqb4eFC9Pv89570+//da+DU07xriUrn+JQt48HWEtLSyxevHi/8WvWpC2E1avTP9uRR6Z/tP60aVN6n+bm9K2zqzFj0gps2rT+fV8bmiJSOGzdCkcckf5+Ghvh4ovT8MEeBzHrjqTHIqKlt3ZDYkthz570bX716vQt7JRT4PDD+/99Dj+8mPna8CTBWWel50uWwKpVaWvqxz9OW3vnnpt2V5oNpCERCk89lY4htLfD8cd7xW31p7K7cvXqzl2U99yTth7+6q/S7j2zgVC3ofDcc+lYwNix6aDx5s1pf/0RR5RdmdnBmzo1PSLgwQfTrqX589O4Cy7Y94wvsyLUXSi0t6dAeOqpFASzZqXTJw87rP+PIZiVRYLzz09bDkuXphMS1q6FU0+FGTPq/2C7DV51Fwq7d6ezhypn1Cxb1nkGjw/M2VBT2XJYuDBtET/4IDz2WDr2cM45MGmStx6sf9VdKFRfPFU573znznTKqdlQNWdO+vnEE2mLYdMm+MlP0imsM2emkyt83MH6Q92FAux7lfCePfDmN+9/AZbZUHTqqennxo2wfHm66n3z5nQF/kknwemnl1uf1b+6C4U9e9KWws6daSvh6KMdCDb8HHFE59bD44+nYw9tbel427x5nVewm/VV3a1OK1fi7t0Lc+fCySeXXZFZuU47LZ2ZtGFDCofbbktbEGYHo+5CAVIgjB/f2T2E2XDX1AQXXpiuiF6zBn76U7jlls4+vMxqVZer1D17fMaFWXfmzEmd7L30Uup87847U/cvZrWqy1DYvduhYHYghx0Gb31r6pp7zZrUqeMtt6Q+wcx6U5eh0NbmTufMenPiiem+Dbt3py2HO++Eu+4quyob7OoyFCZOTL2gmlnvzjknHYxesyZ1unfLLZ03ljLrqu5CISIdaDaz2o0fn7rk3rYtHWu49VZ44YWyq7LBqO5CwcwOjpROXW1uTqeu3ncfPPJI2VXZYFN3F6+B+zgyOxQtLelU1Yce6rytauW+DmZ1uaXgUDA7NIcdli7+fOml1Knkb39bdkU2WNRdKFTf89fMDl5zM5x9dupgb+nSdMc3s7oLBei8abyZHZpx49I1DevXpzOTbr89ffGy4asuQ8FdW5j1n4YGeNvbUs+r69enLrl9yurwVejqVdLFkpZLWiHp2m6mv1rS/ZKWSHpS0qVF1mNm3ZPg0ks7e1q97bZ0vMGGn8JCQVIDcD1wCTALuELSrC7NPgfMj4jZwOXAt2qZ96hR/VmpmVWce266Wc/q1XD33bBuXdkV2UArckvhDGBFRKyMiD3AD4B5XdoEMD57PgFYU8uMm5v7rUYz6+L009Od3NauhZ/9zLuShpsiQ2EqsKpquDUbV+064EpJrcDdwEe6m5GkayQtlrR469ZNPvvIrGCvehXMmpV2Id16q4NhOCkyFLpbdXc9r+EK4LsRMQ24FLhZ0n41RcSNEdESES1jxx7OuHEFVGtm+5g2LZ3pt359OivJhociQ6EVOLZqeBr77x66GpgPEBGPAM3A5AJrMrM+OP/8dAOftWth/vyyq7GBUGQoLAJmSjpOUhPpQPKCLm3+BLwFQNLrSKHgGwmaDSJvehPs2JEOOt97b9nVWNEKC4WIaAM+DNwDPEM6y+hpSV+QdFnW7JPA+yX9HrgNuCrCl86YDTYXXQSbNqWeVTdvLrsaK5LqbR08fXpL3HDDYqZMKbsSs+Fl27bUR9Ixx8C73+2LSOuNpMcioqW3dnX3a5X8x2hWhrFj4bWvTWck3XFH2dVYUbx6NbOaTZ+ebu+5YQP86ldlV2NFcCiYWZ9cdFHqJ+n559153lDkUDCzPmlsTFc8b9gACxY4GIYah4KZ9dnUqal31eefhx/+sOxqrD/VZSj4m4lZ+ebOhfZ2aG2FX/yi7Gqsv9RlKPjsI7PB4bzzYOvW1N32o4+WXY31B69ezeygSXDhhen4wtKlacvB6ptDwcwOSVNT54Hnu+4quxo7VA4FMztkxxyTthLWrElXPVv9ciiYWb847zx4+WVYvhy2bCm7GjtYDgUz6xfNzfDGN6Zutn/+87KrsYNVl6HgU1LNBqdJk9Jd2tavh4ceKrsaOxh1GQo+JdVs8LrwwtTN9ooVsHdv2dVYX3n1amb9qqkJTjst3ZTnvvvKrsb6yqFgZv2ucr+TF15IWwxWPxwKZlaIc85JZyM9/HDZlVhfOBTMrBDNzanTvFde8dlI9cShYGaFectb0m08//SndHMeG/wcCmZWGAnmzElnIz3wQNnVWC0cCmZWqIkTUxcYzz0HL75YdjXWG4eCmRXurLPSsYXHHy+7EutNXYZCc3PZFZhZX4wbl650XrfOPRIMdnUZCiNHll2BmfXVmWem7i/uv7/sSqwndRkKZlZ/Jk6EPXtg9eqyK7GeOBTMbEBI8IY3pJvx+EykwcuhYGYDZsoU2L4dNm8uuxI7EIeCmQ2YxsZ0THD1ali4sOxqrDsOBTMbUOeck05PbW0tuxLrjkPBzAZUczOMGpUuZHNneYOPQ8HMBtzZZ6fjCr7CefBprLWhpKnA9OrXRMSDRRRlZkNbUxNMnpxC4bHH4PTTy67IKmraUpD0ZeBh4HPAp7PHp2p43cWSlktaIenaA7R5p6Rlkp6WdGsfajezOvb618PWrbBqVdmVWLVatxT+HHhtRNTc+a2kBuB64K1AK7BI0oKIWFbVZibwt8DZEbFR0pTaSzezejZmDHR0pKucbfCo9ZjCSqCvnUucAayIiJURsQf4ATCvS5v3A9dHxEaAiFjXx/cwszp22mmwZYsvZhtMat1S2AE8Iek+IN9aiIiP9vCaqUD1hmErcGaXNicCSHoYaACuiwjfo8lsmJg0CXbs8NbCYFJrKCzIHn2hbsZ17R+xEZgJzAWmAQ9JOikiNu0zI+ka4BqASZOO72MZZjZYNTbC+PGwdi2sWAEnnFB2RVZTKETE9yQ1kX2zB5ZHxN5eXtYKHFs1PA1Y002bhdm8npO0nBQSi7q8/43AjQAzZrS4412zIeSkk+DRR9MtOx0K5av17KO5wH+QDhx/C/iDpPN6edkiYKak47JAuZz9tzZ+BFyQvcdkUuisrLl6M6t7Eyakey2s6fqV0UpR64HmrwEXRcT5EXEe8Dbg6z29ICLagA8D9wDPAPMj4mlJX5B0WdbsHuBlScuA+4FPR8TLB7MgZlafJDj5ZHj5ZV/hPBjUekxhZEQsrwxExB8k9Xo2UkTcDdzdZdznq54H8InsYWbD1NFHw9KlqVttK1etobBY0k3Azdnwu4DHiinJzIabxkY46qi0C2nbNhg7tuyKhq9adx99EHga+CjwMWAZ8IGiijKz4eeEE9I1C488UnYlw1utZx/tBv4pe5iZ9btx49LtOl94oexKhrcetxQkzc9+PiXpya6PgSnRzIaLSn9Iv/td2ZUMX71tKXws+/n2ogsxM5syBZ55Jp2JZOXocUshIiq9nW8AVkXEC8Ao4A3sfyGamdkhGT06dZLns5DKU+uB5geB5uyeCvcB7wO+W1RRZjZ8zZ6dthR8D+dy1BoKiogdwF8C34iIvwBmFVeWmQ1Xhx8O27d7a6EsNYeCpLNI1yfclY2r+a5tZma1Gj0aIuCVV8quZHiqNRQ+TroZzr9nXVUcT+qWwsys382aBRs3+iykMtR6ncIDwANVwytJF7KZmfW7KVPg2Wd9FlIZegwFSf8nIj4u6Sfsfy8EIuKybl5mZnZIxoxJXV+sXl12JcNPb1sKlb6O/nfRhZiZVZs9G558Eu6/Hy64oOxqho8eQyEiKp3eLQZ2RkQHgKQG0vUKZmaFmDABdu5MHeTZwKn1QPN9wJiq4dHAvf1fjplZ0tiYHmvXll3J8FJrKDRHRJ7X2fMxPbQ3Mztks2alnlN9852BU2sobJd0WmVA0unAzmJK6p1U1jub2UCaMiXtQnrxxd7bWv+o9QK0jwN3SKr0d3Q08NfFlGRmljQ1QUNDurp51y5obi67oqGv1usUFkn6M+C1gIBnI2JvoZWZmQGnnw5PPAGLFsG555ZdzdBX0+4jSWOA/wF8LCKeAmZIcnfaZla4ww5LN9/ZtKnsSoaHWo8pfAfYA5yVDbcCXyykIjOzKs3NqS+kjRvLrmR4qDUUXhMRXwH2AkTETtJuJDOzws2enY4r/OY3ZVcy9NUaCnskjSbr6kLSa4DdhVVlZlZl0qR0FlJra9mVDH21nn3098DPgWMl/RtwNnBVUUWZmVVraursTrujA0bU+nXW+qzXj1aSgGdJN9i5CrgNaImIXxdamZlZlVNPTVsLixeXXcnQ1uuWQkSEpB9FxOl03mDHzGxAjR+frlXYvr3sSoa2WjfCFkp6Y6GVmJn1YMwY35FtINR6TOEC4AOSnge2k848iog4pajCzMyqSTB9ejoLKcLd3RSl1lC4pNAqzMxqcNRRsH59uphtlDvvL0Rvd15rBj4AnAA8BdwUEW0DUZiZWVcTJ0J7u88+KlJvH+33gBZSIFwCfK3wiszMrDS97T6aFREnA0i6CXi0+JLMzKwsvW0p5D2hereRmdnQ11sovEHSluyxFTil8lzSlt5mLuliScslrZB0bQ/t3iEpJLX0dQHMzKz/9Lj7KCIaDnbGkhqA64G3knpVXSRpQUQs69JuHPBR4HcH+15mZtY/ijyGfwawIiJWRsQe4AfAvG7a/SPwFWBXgbWYmVkNigyFqcCqquHWbFxO0mzg2Ij4aU8zknSNpMWSFm/b5k7VzcyKUmQodHe9YeQTpRHA14FP9jajiLgxIloiomXs2CP6sUQzM6tWZCi0AsdWDU8D1lQNjwNOAn6ddZ8xB1jgg81mZuUpMhQWATMlHSepCbgcWFCZGBGbI2JyRMyIiBnAQuCyiHDHuGZmJSksFLLrGj4M3AM8A8yPiKclfUHSZUW9r5mZHbxaO8Q7KBFxN3B3l3GfP0DbuUXWYmZmvXO3UmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUKDQVJF0taLmmFpGu7mf4JScskPSnpPknTi6zHzMx6VlgoSGoArgcuAWYBV0ia1aXZEqAlIk4B7gS+UlQ9ZmbWuyK3FM4AVkTEyojYA/wAmFfdICLuj4gd2eBCYFqB9ZiZWS+KDIWpwKqq4dZs3IFcDfysuwmSrpG0WNLibds29mOJZmZWrchQUDfjotuG0pVAC/DV7qZHxI0R0RIRLWPHHtGPJZqZWbXGAufdChxbNTwNWNO1kaQLgc8C50fE7gLrMTOzXhS5pbAImCnpOElNwOXAguoGkmYD/wpcFhHrCqzFzMxqUFgoREQb8GHgHuAZYH5EPC3pC5Iuy5p9FRgL3CHpCUkLDjA7MzMbAEXuPiIi7gbu7jLu81XPLyzy/c3MrG98RbOZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUKDQVJF0taLmmFpGu7mT5K0u3Z9N9JmlFkPWZm1rPCQkFSA3A9cAkwC7hC0qwuza4GNkbECcDXgS8XVY+ZmfWuscB5nwGsiIiVAJJ+AMwDllW1mQdclz2/E/imJEVE9DTjXbugscjKzWzQam8vu4KhrchV61RgVdVwK3DmgdpERJukzcAkYEN1I0nXANdkQ3vnzh3/R+g5OIauvUfAyI1lV1Ge4bz8w3nZoXP5I2D7zrKrKcFkuqwb+2h6LY2KDAV1M67riryWNkTEjcCNAJIWR2xpOfTy6lNa/l1e/mFoOC87ePnT8kfhy1/kgeZW4Niq4WnAmgO1kdQITABeKbAmMzPrQZGhsAiYKek4SU3A5cCCLm0WAO/Nnr8D+FVvxxPMzKw4he0+yo4RfBi4B2gAvh0RT0v6ArA4IhYANwE3S1pB2kK4vIZZ31hUzXXCyz98DedlBy//gCy//MXczMwqfEWzmZnlHApmZpYbtKEw3LvIqGH5PyFpmaQnJd0nqaZzkOtBb8te1e4dkkLSkDpNsZbll/TO7Pf/tKRbB7rGItXwt/9qSfdLWpL9/V9aRp1FkPRtSeskLT3AdEn65+yzeVLSaf1eREQMugfpwPQfgeOBJuD3wKwubf4bcEP2/HLg9rLrHuDlvwAYkz3/4FBZ/lqWPWs3DngQWAi0lF33AP/uZwJLgCOy4Sll1z3Ay38j8MHs+Szg+bLr7sflPw84DVh6gOmXAj8jXeM1B/hdf9cwWLcU8i4yImIPUOkio9o84HvZ8zuBt0jq7mK4etTr8kfE/RGxIxtcSLoOZCio5XcP8I/AV4BdA1ncAKhl+d8PXB8RGwEiYt0A11ikWpY/gPHZ8wnsf/1T3YqIB+n5Wq15wPcjWQgcLuno/qxhsIZCd11kTD1Qm4hoAypdZAwFtSx/tatJ3x6Ggl6XXdJs4NiI+OlAFjZAavndnwicKOlhSQslXTxg1RWvluW/DrhSUitwN/CRgSltUOjruqHPBmu3cv3WRUadqnnZJF0JtADnF1rRwOlx2SWNIPWoe9VAFTTAavndN5J2Ic0lbSE+JOmkiNhUcG0DoZblvwL4bkR8TdJZpGudToqIjuLLK13h673BuqUw3LvIqGX5kXQh8FngsojYPUC1Fa23ZR8HnAT8WtLzpP2qC4bQweZa//Z/HBF7I+I5YDkpJIaCWpb/amA+QEQ8AjSTOosbDmpaNxyKwRoKw72LjF6XP9uF8q+kQBhK+5R7XPaI2BwRkyNiRkTMIB1PuSwiFpdTbr+r5W//R6QTDZA0mbQ7aeWAVlmcWpb/T8BbACS9jhQK6we0yvIsAN6TnYU0B9gcES/25xsMyt1HUVwXGXWhxuX/KjAWuCM7vv6niListKL7SY3LPmTVuPz3ABdJWga0A5+OiJfLq7r/1Lj8nwT+n6T/Ttp1ctVQ+UIo6TbSbsHJ2TGTvwdGAkTEDaRjKJcCK4AdwPv6vYYh8lmamVk/GKy7j8zMrAQOBTMzyzkUzMws51AwM7OcQ8HMzHIOBbMuJLVLekLSUkk/kXR4P8//KknfzJ5fJ+lT/Tl/s0PhUDDb386IODUiTiJdA/OhsgsyGygOBbOePUJVh2OSPi1pUdaX/T9UjX9PNu73km7Oxv2n7F4fSyTdK+lVJdRv1ieD8opms8FAUgOpO4WbsuGLSH0MnUHqmGyBpPOAl0l9UJ0dERskTcxm8RtgTkSEpP8KfIZ0Na7ZoOVQMNvfaElPADOAx4BfZuMvyh5LsuGxpJB4A3BnRGwAiIhKx4zTgNuz/u6bgOcGpHqzQ+DdR2b72xkRpwLTSSvzyjEFAf8rO95wakScEBE3ZeO76y/mG8A3I+Jk4G9IHbeZDWoOBbMDiIjNwEeBT0kaSeqk7b9IGgsgaaqkKcB9wDslTcrGV3YfTQBWZ8/fi1kd8O4jsx5ExBJJvwcuj4ibs66aH8l6pt0GXJn14vkl4AFJ7aTdS1eR7hB2h6TVpC6+jytjGcz6wr2kmplZzruPzMws51AwM7OcQ8HMzHIOBTMzyzkUzMws51AwM7OcQ8HMzHL/H8Ea5x0Iz0fnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "file_prefix = 'cct_images/'\n",
    "\n",
    "results_path = '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/6704_detections_train_ss1_val_cct20_20190908092921.json'\n",
    "\n",
    "gt_db_path = '/beaver_disk/camtrap/caltech/benchmark/cct-20/caltech-20_bboxes_20190904_val.json'\n",
    "\n",
    "detection_res = make_detection_res(results_path, file_prefix=file_prefix)\n",
    "\n",
    "gt_indexed = get_gt_db(gt_db_path)\n",
    "\n",
    "per_image_gts, per_image_detections = detector_eval.get_per_image_gts_and_detections(gt_indexed, detection_res)\n",
    "\n",
    "per_cat_metrics = detector_eval.compute_precision_recall_bbox(per_image_detections, per_image_gts, num_gt_classes, \n",
    "                                                         matching_iou_threshold=0.5)\n",
    "\n",
    "show_metrics(per_cat_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## empty vs non-empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9071859347848308\n"
     ]
    }
   ],
   "source": [
    "accuracy = detector_eval.compute_emptiness_accuracy(gt_indexed, detection_res, threshold=empty_threshold)\n",
    "\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on CCT-20, val on CCT-20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 162/17918 [00:00<00:10, 1615.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running per-object analysis...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17918/17918 [00:11<00:00, 1617.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing precision recall for each category...\n",
      "Number of ground truth in category 1 is 18539\n",
      "Number of ground truth in category 2 is 0\n",
      "\n",
      "one_class\n",
      "Number of gt: 18539\n",
      "Average precision 0.9346375483403553\n",
      "\n",
      "1\n",
      "Number of gt: 18539\n",
      "Average precision 0.9348238623588044\n",
      "\n",
      "2\n",
      "Number of gt: 0\n",
      "Average precision nan\n",
      "\n",
      "mAP as the average of AP across the 2 categories is 0.4674\n",
      "Precision at 0.8 recall: (0.9758536745838542, 0.9990000128746033)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG6FJREFUeJzt3XucX3V95/HXe2YyExLCBEgQSJBBCUiWu+EmKrhQBOqStg8eCBUVl5XKguhqdWl1lVJ97KJrb4q16YIXLCCwLZvatNQiBa0EEoqgRNAYLhnugRBynUwmn/3je86ZXyZzOZP8zu83l/fz8fg9fuf++5zfJOf9O99zU0RgZmYG0NLsAszMbOxwKJiZWcGhYGZmBYeCmZkVHApmZlZwKJiZWcGhYBOOpKcknTnEuNMldTeghksk/bjiz+iSFJLaqvwcm1wcCmZmVnAomJlZwaFgE9UJklZIWivpm5Km1o6U9IeS1mRNTe+rGd4p6TuSXpb0tKTPSmrJxv2lpDtqpr1O0t2SNFIxkt4i6QeSXpX0hKQLsuEnS3pBUmvNtL8t6dGsu0XS1ZJ+LekVSbdJ2mf3vx6zwTkUbKJ6H/Bu4M3AYcBna8btD8wC5gAfBBZJOjwb91WgE3gTcBrwAeBD2bhPAkdnxwveAVwKfDBGuFeMpOnAD4Cbgf2Ai4CvS/oPEbEU2Aj8x5pZfjebFuAq4LeyWg4E1gLXl/8azEbHoWAT1dciYnVEvAp8kbQhrvU/IqInIu4F/gG4IPu1/l7gDyJifUQ8BXwFeD9ARGwCLgb+BPgu8NGIKHPQ+j3AUxHxzYjYFhH/Dvxf4Pxs/C15fZJmAOdmwwB+D/hMRHRHRA9wDXC+Dy5bVfwPyyaq1TXdT5N+ZefWRsTGQcbPAtqz/tpxc/KeiHhQ0irSL/7bStZyMHCSpNdqhrUBN2XdNwM/kXQ58DvAv0fE0zXz/p2k7TXz9gFvKPnZZqPiPQWbqA6q6X4j8FxN/95Zk87A8WuAXtKGuHbcs3mPpCuAjmz6T5esZTVwb0TMrHntGRGXA0TEClL4nMOOTUf5vOcMmHdqRDy706eY1YFDwSaqKyTNzQ7K/iHwvQHj/0hSe3Zs4D3A7RHRR/r1/0VJMyQdDHyC1FSEpMOAL5CakN4PfFrSsSVq+T5wmKT3S5qSvU6QdETNNDeTjh+8E7i9Zvg3snoOzmqYLWnhqL4Js1FwKNhEdTPwz8Cq7PWFmnEvkA7YPgf8DfCRiHg8G/dR0oHfVcCPs+XcmLXhfxe4LiIeiYhfkcLmJkkdwxUSEeuBs4ALs898AbiOtMeRuwU4HfhhRKypGf7nwGLgnyWtB5YCJ5X/GsxGR37IjpmZ5bynYGZmBYeCmZkVHApmZlZwKJiZWWHcXbw2a9as6OrqanYZZmbjykMPPbQmImaPNN24C4Wuri6WL1/e7DLMzMYVSU+PPJWbj8zMrIZDwczMCg4FMzMrOBTMzKzgUDAzs4JDwczMCpWFgqQbJb0k6edDjJekv5C0UtKjko6vqhYzMyunyj2FbwFnDzP+HGBe9roM+MuyC96+Pb18g1czs/qq7OK1iLhPUtcwkywEvpM99HyppJmSDoiI54db7oYN8KMfpe6WFjjqqNQ9c2YdijYzm+SaeUXzHHZ8jm53NmynUJB0GWlvgtmzu1i9GrZuhbVr4emnoa8P9t8fTj4Ztm2DzZvhxRfTvF1dqfuAA6C3N41rbYVp09J7Z2eav7c3zbtuHey1Vwqfzs7+PZLt29N0eb80sMb+7p6e9D5jRlpme3sKsHzefA9HSq++vjS+paX/s3b+Dob+IocaN5p5IqCtLdUwcHzePfDdzCaeZobCYJuWQRuEImIRsAjg8MMXxLx5afhDD6WN/AsvpA3rnXemDVZvb3pNmQIrVqQNc1tbeu/rS/N2dKSNb2vrwM9Kyxi44c831vkGW0rdtRv72vlaWtJnDmVg09dgG+naZUakWvPugf21n1277JYKGwgHC4fBwqS2f9u21D1zZgrmqVPT36KlJf3N8u+0NiTb29Pfbdu29GOgpye9t7SkcXmwTp8O++yT+l97LY3fZ58da8mDuKcHXn89/ZgYWO9g4dfTkz6/s7N//ilT0ucPNW/+t8n/HQz13YzUndfsMLZGaGYodLPjw9XnsuPD1YfV1gYnZQ8lXL8eHnss/Qdta+t/zzce+S/8tra0Ie3pgS1b0l7B1Kmwxx5pORFpXEdHem9v33HDm2+ocnl3/l67Uc73PHp6+n+B5xu5fNpcvoGbMmXXj5MMNV8eXgM3VrXjI/q/o/b2/vWoDb58nu3b017UjBlpvWqXVRtYgwVTHqqbNqXvP//8odYlr602eGs/p3bezZvTdHvs0b/Rlvo3yAO/n9r5a7+T4TbA0o4/IoYLxSo34LXfS2tr+ve6777p1daW/j9Af0BKsGZN+ptNn94/LP9h0dIydCDm7729afr8/8rAwIL+f2dtbTt+j/m/fYfa+NDMUFgMXCnpVtIzZ9eNdDxhKDNmpKYjm9zyUN+yJYXc+vUpmPO9jZYW2Lhx8IAYrLu2Pw+iPJy2bdtx4zrYPNu3p+k6Ovo34gP3PvP32uXUht7GjWmPKg/m2vm3bEk/arZuTc2o06f3D4c0Ll9m3kRZGyiw48Z7sGbRgXuag4XBaLS2plpmz06vPEg6O9OPtFmz0o+jvr7UjNvXl0Kmo6P/R51Vq7JQkJQ/iHyWpG7g88AUgIj4BrAEOBdYCWwCPlRVLTY5SGlDOHVq6t933+bW0wx5aPX1pUDZsiX9yt+4ccfpakMuNzCYoL+5deCxtMH2TPOgrN2DzINt+vS0jHzP+ZVX4Kmn+veSoX/PJ997yUNssCbQ2bNhv/3SMmfO7G8KbG9Pf//29hQuNnpVnn100QjjA7iiqs83m4zyDWhra//xjrEsIgVDBHR3p9DI97DyE0DyZsc99khNl3196f2Xv+wPksECBVKoHHdcak2YPbv/B4MNbdw9T8HMJg4pNQ0BHHro6OaNSE1Ozz/fv3fU29v/vnlzapa6554UkPkJDUcckZqm5sxJTXO2I4eCmY1LUmo6GukapQ0b4Fe/SgGydWt67+hIzUutrfCOd6SAGO5swcnEX4OZTWh77pmakHJr1qSmqvXrU3PVXXelpqnDD4cTTvBZUg4FM5tUZs1KL0jNTsuXp5DYtCkdp/jN34S9925ujc3kUDCzSUtKewcA998Pzz0Hf/u3qXnpggvGx8H6evOts83MgFNOSccX1qxJp8zecku6a8Jk41AwM8u0t8OZZ6brKp59NoXCd7/bf73GZOBQMDMb4Ljj4Kyz+vcabr558gSDQ8HMbBASvPvd6WD0iy+mYJgMz3BxKJiZDeMd70gXvb30Ujp9daJzKJiZjeC009IV0k8+Cffe2+xqquVQMDMr4Ywz0vGFVatSOExUDgUzsxLa22HBgnSvpQcfnLjHFxwKZmYlzZ6dbofR3Q1LljS7mmo4FMzMRqGrK93a+8kn4YEHml1N/TkUzMxG6e1vT3dbfa70A4THD4eCmdkoTZ2abtn9zDPwwx82u5r6ciiYme2Ck05Kd1ZduRJefbXZ1dSPQ8HMbBdIcOqp6TTVn/yk2dXUj0PBzGwXTZ+enuC2evXEOejsUDAz2w1ve1tqPnrxxWZXUh8OBTOz3dDRkfYYnnkGXnih2dXsPoeCmdluOuaY9MznBx9sdiW7z6FgZrab9torvXd3p3AYzxwKZmZ1cPzxKRDuv7/Zleweh4KZWR3MmpVOU33ySXjttWZXs+scCmZmdXLiibB2Ldx9d7Mr2XUOBTOzOpkxA3p701PaenqaXc2ucSiYmdXR298Or78O99zT7Ep2jUPBzKyO8jORnnpqfF7Q5lAwM6uzt70tHWwej2ciORTMzOps2jRobU3PW1i7ttnVjE6loSDpbElPSFop6epBxr9R0j2SHpb0qKRzq6zHzKxRjj02Xbfw6KPNrmR0KgsFSa3A9cA5wHzgIknzB0z2WeC2iDgOuBD4elX1mJk10l57pTORxttxhSr3FE4EVkbEqojYCtwKLBwwTQDZYRk6gQn4cDszm4za2qC9PTUfbdvW7GrKqzIU5gCra/q7s2G1rgEultQNLAE+OtiCJF0mabmk5evWvVxFrWZmdfeWt8CGDfDQQ82upLwqQ0GDDIsB/RcB34qIucC5wE2SdqopIhZFxIKIWNDZObuCUs3M6m/mTNiyZXw9rrPKUOgGDqrpn8vOzUOXArcBRMT9wFRgVoU1mZk1zPTpsH07PP98syspr8pQWAbMk3SIpHbSgeTFA6Z5BjgDQNIRpFBw+5CZTRhHHZXOQnr88WZXUk5loRAR24ArgbuAX5DOMnpM0rWSzssm+yTwYUmPALcAl0TEwCYmM7Nxa889UxPSK680u5Jy2qpceEQsIR1Arh32uZruFcCpVdZgZtZM06alU1PXrWt2JeX4imYzswp1dEDE+HnGgkPBzKxic+d6T8HMzDLTpsHWrbB69cjTNptDwcysYvvtB5s3p0d1jnUOBTOzirW3p4PNmzY1u5KRORTMzCrW0ZHeN2xobh1lOBTMzComwYEHjo8zkBwKZmYNcMAB6dnNDz7Y7EqG51AwM2uA6dOhpwc2bmx2JcNzKJiZNcC0aem5CmP98ZwOBTOzBmhpgb33HvtnIDkUzMwaZP/90x1TN29udiVDq/SGeGZm1u/AA9N9kDTYI8jGCO8pmJk1SEcHzJjhUDAzs3HCoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVmh9OM4Jc0BDq6dJyLuq6IoMzNrjlKhIOk64L3ACqAvGxzAsKEg6Wzgz4FW4P9ExP8aZJoLgGuy5T0SEb9btngzM6uvsnsKvwUcHhE9ZRcsqRW4HvgNoBtYJmlxRKyomWYe8AfAqRGxVtJ+5Us3M7N6K3tMYRUwZZTLPhFYGRGrImIrcCuwcMA0Hwauj4i1ABHx0ig/w8zM6qjsnsIm4KeS7gaKvYWIuGqYeeYAq2v6u4GTBkxzGICkfyM1MV0TEf9UsiYzM6uzsqGwOHuNhgYZFoN8/jzgdGAu8CNJR0bEazssSLoMuAzgDW944yjLMDOzskqFQkR8W1I72S974ImI6B1htm7goJr+ucBzg0yzNFvWk5KeIIXEsgGfvwhYBHD44QsGBouZmdVJqWMKkk4HfkU6cPx14JeS3jnCbMuAeZIOyQLlQnbe27gTeFf2GbNIobOqdPVmZlZXZZuPvgKcFRFPAEg6DLgFeOtQM0TENklXAneRjhfcGBGPSboWWB4Ri7NxZ0nKT3X9VES8suurY2Zmu6NsKEzJAwEgIn4pacSzkSJiCbBkwLDP1XQH8InsZWZmTVY2FJZLugG4Ket/H/BQNSWZmVmzlA2Fy4ErgKtIZxXdRzq2YGZmE0jZs496gD/JXmZmNkENGwqSbouICyT9jJ2vMSAijq6sMjMza7iR9hQ+lr2/p+pCzMys+Ya9TiEins861wCrI+JpoAM4hp0vRDMzs3Gu7A3x7gOmZs9UuBv4EPCtqooyM7PmKBsKiohNwO8AX42I3wbmV1eWmZk1Q+lQkHQK6fqEf8iGlX5qm5mZjQ9lQ+HjpIfh/F12q4o3AfdUV5aZmTVD2esU7gXurelfRbqQzczMJpCRrlP4s4j4uKS/Z/DrFM6rrDIzM2u4kfYU8nsd/e+qCzEzs+YbNhQiIr/p3XJgc0RsB5DUSrpewczMJpCyB5rvBqbV9O8B/Ev9yzEzs2YqGwpTI2JD3pN1TxtmejMzG4fKhsJGScfnPZLeCmyupiQzM2uWshegfRy4XVJ+v6MDgPdWU5KZmTVL2esUlkl6C3A46SE7j0dEb6WVmZlZw5VqPpI0DfjvwMci4mdAlyTfTtvMbIIpe0zhm8BW4JSsvxv4QiUVmZlZ05QNhTdHxJeAXoCI2ExqRjIzswmkbChslbQH2a0uJL0Z6KmsKjMza4qyZx99Hvgn4CBJfwOcClxSVVFmZtYcI4aCJAGPkx6wczKp2ehjEbGm4trMzKzBRgyFiAhJd0bEW+l/wI6ZmU1AZY8pLJV0QqWVmJlZ05U9pvAu4COSngI2kpqQIiKOrqowMzNrvLKhcE6lVZiZ2Zgw0pPXpgIfAQ4FfgbcEBHbGlGYmZk13kjHFL4NLCAFwjnAVyqvyMzMmmak5qP5EXEUgKQbgAerL8nMzJplpD2F4k6objYyM5v4RgqFYyS9nr3WA0fn3ZJeH2nhks6W9ISklZKuHma68yWFpAWjXQEzM6ufYZuPIqJ1VxcsqRW4HvgN0l1Vl0laHBErBkw3A7gKeGBXP8vMzOqj7MVru+JEYGVErIqIrcCtwMJBpvtj4EvAlgprMTOzEqoMhTnA6pr+7mxYQdJxwEER8f3hFiTpMknLJS1ft+7l+ldqZmZAtaEw2PMWohgptQB/CnxypAVFxKKIWBARCzo7Z9exRDMzq1VlKHQDB9X0zwWeq+mfARwJ/Gt2+4yTgcU+2Gxm1jxVhsIyYJ6kQyS1AxcCi/OREbEuImZFRFdEdAFLgfMiYnmFNZmZ2TAqC4XsuoYrgbuAXwC3RcRjkq6VdF5Vn2tmZruu7A3xdklELAGWDBj2uSGmPb3KWszMbGRVNh+Zmdk441AwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzKzgUzMys4FAwM7OCQ8HMzAoOBTMzK1QaCpLOlvSEpJWSrh5k/CckrZD0qKS7JR1cZT1mZja8ykJBUitwPXAOMB+4SNL8AZM9DCyIiKOBO4AvVVWPmZmNrMo9hROBlRGxKiK2ArcCC2sniIh7ImJT1rsUmFthPWZmNoIqQ2EOsLqmvzsbNpRLgX8cbISkyyQtl7R83bqX61iimZnVqjIUNMiwGHRC6WJgAfDlwcZHxKKIWBARCzo7Z9exRDMzq9VW4bK7gYNq+ucCzw2cSNKZwGeA0yKip8J6zMxsBFXuKSwD5kk6RFI7cCGwuHYCSccBfwWcFxEvVViLmZmVUFkoRMQ24ErgLuAXwG0R8ZikayWdl032ZWBP4HZJP5W0eIjFmZlZA1TZfERELAGWDBj2uZruM6v8fDMzGx1f0WxmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZgWHgpmZFRwKZmZWcCiYmVnBoWBmZoVKQ0HS2ZKekLRS0tWDjO+Q9L1s/AOSuqqsx8zMhldZKEhqBa4HzgHmAxdJmj9gskuBtRFxKPCnwHVV1WNmZiNrq3DZJwIrI2IVgKRbgYXAipppFgLXZN13AF+TpIiI4Ra8ZQu0VVm5mVlFenubXcHwqty0zgFW1/R3AycNNU1EbJO0DtgXWFM7kaTLgMuyvq2nn77Xr2HY3JjAeveGKWubXUXzTOb1n8zrDhNr/Tds2oWZZjFg2zhKB5eZqMpQ0CDDBm7Jy0xDRCwCFgFIWh7x+oLdL298Suu/xes/CU3mdQevf1r/qHz9qzzQ3A0cVNM/F3huqGkktQGdwKsV1mRmZsOoMhSWAfMkHSKpHbgQWDxgmsXAB7Pu84EfjnQ8wczMqlNZ81F2jOBK4C6gFbgxIh6TdC2wPCIWAzcAN0laSdpDuLDEohdVVfM44fWfvCbzuoPXvyHrL/8wNzOznK9oNjOzgkPBzMwKYzYUJvstMkqs/yckrZD0qKS7JZU6B3k8GGnda6Y7X1JImlCnKZZZf0kXZH//xyTd3Ogaq1Ti3/4bJd0j6eHs3/+5zaizCpJulPSSpJ8PMV6S/iL7bh6VdHzdi4iIMfciHZj+NfAmoB14BJg/YJr/Cnwj674Q+F6z627w+r8LmJZ1Xz5R1r/MumfTzQDuA5YCC5pdd4P/9vOAh4G9s/79ml13g9d/EXB51j0feKrZdddx/d8JHA/8fIjx5wL/SLrG62TggXrXMFb3FIpbZETEViC/RUathcC3s+47gDMkDXYx3Hg04vpHxD0RkV8VuZR0HchEUOZvD/DHwJeALY0srgHKrP+HgesjYi1ARLzU4BqrVGb9A9gr6+5k5+ufxq2IuI/hr9VaCHwnkqXATEkH1LOGsRoKg90iY85Q00TENiC/RcZEUGb9a11K+vUwEYy47pKOAw6KiO83srAGKfO3Pww4TNK/SVoq6eyGVVe9Mut/DXCxpG5gCfDRxpQ2Jox22zBqY/W2cnW7RcY4VXrdJF0MLABOq7Sixhl23SW1kO6oe0mjCmqwMn/7NlIT0umkPcQfSToyIl6ruLZGKLP+FwHfioivSDqFdK3TkRGxvfrymq7y7d5Y3VOY7LfIKLP+SDoT+AxwXkT0NKi2qo207jOAI4F/lfQUqV118QQ62Fz23/7/i4jeiHgSeIIUEhNBmfW/FLgNICLuB6aSbhY3GZTaNuyOsRoKk/0WGSOuf9aE8lekQJhIbcrDrntErIuIWRHRFRFdpOMp50XE8uaUW3dl/u3fSTrRAEmzSM1JqxpaZXXKrP8zwBkAko4ghcLLDa2yeRYDH8jOQjoZWBcRz9fzA8Zk81FUd4uMcaHk+n8Z2BO4PTu+/kxEnNe0ouuk5LpPWCXX/y7gLEkrgD7gUxHxSvOqrp+S6/9J4K8l/TdS08klE+UHoaRbSM2Cs7JjJp8HpgBExDdIx1DOBVYCm4AP1b2GCfJdmplZHYzV5iMzM2sCh4KZmRUcCmZmVnAomJlZwaFgZmYFh4LZAJL6JP1U0s8l/b2kmXVe/iWSvpZ1XyPp9+u5fLPd4VAw29nmiDg2Io4kXQNzRbMLMmsUh4LZ8O6n5oZjkj4laVl2L/s/qhn+gWzYI5Juyob9p+xZHw9L+hdJb2hC/WajMiavaDYbCyS1km6ncEPWfxbpHkMnkm5MtljSO4FXSPegOjUi1kjaJ1vEj4GTIyIk/Rfg06Srcc3GLIeC2c72kPRToAt4CPhBNvys7PVw1r8nKSSOAe6IiDUAEZHfmHEu8L3sfvftwJMNqd5sN7j5yGxnmyPiWOBg0sY8P6Yg4H9mxxuOjYhDI+KGbPhg94v5KvC1iDgK+D3SjdvMxjSHgtkQImIdcBXw+5KmkG7S9p8l7QkgaY6k/YC7gQsk7ZsNz5uPOoFns+4PYjYOuPnIbBgR8bCkR4ALI+Km7FbN92d3pt0AXJzdxfOLwL2S+kjNS5eQnhB2u6RnSbf4PqQZ62A2Gr5LqpmZFdx8ZGZmBYeCmZkVHApmZlZwKJiZWcGhYGZmBYeCmZkVHApmZlb4/2Ggu42+TEi/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "file_prefix = 'cct_images/'\n",
    "\n",
    "results_path = '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/8506_detections_train_cct_20_val_cct20_20190908093026.json'\n",
    "\n",
    "gt_db_path = '/beaver_disk/camtrap/caltech/benchmark/cct-20/caltech-20_bboxes_20190904_val.json'\n",
    "\n",
    "detection_res = make_detection_res(results_path, file_prefix=file_prefix)\n",
    "\n",
    "gt_indexed = get_gt_db(gt_db_path)\n",
    "\n",
    "per_image_gts, per_image_detections = detector_eval.get_per_image_gts_and_detections(gt_indexed, detection_res)\n",
    "\n",
    "per_cat_metrics = detector_eval.compute_precision_recall_bbox(per_image_detections, per_image_gts, num_gt_classes, \n",
    "                                                         matching_iou_threshold=0.5)\n",
    "\n",
    "show_metrics(per_cat_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## empty vs non-empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9010528467750178\n"
     ]
    }
   ],
   "source": [
    "accuracy = detector_eval.compute_emptiness_accuracy(gt_indexed, detection_res, threshold=empty_threshold)\n",
    "\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on CCT-20, val on SS S1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 157/2725 [00:00<00:01, 1562.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running per-object analysis...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2725/2725 [00:01<00:00, 1574.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing precision recall for each category...\n",
      "Number of ground truth in category 1 is 4208\n",
      "Number of ground truth in category 2 is 7\n",
      "\n",
      "one_class\n",
      "Number of gt: 4215\n",
      "Average precision 0.7444823251535193\n",
      "\n",
      "1\n",
      "Number of gt: 4208\n",
      "Average precision 0.7457207700860465\n",
      "\n",
      "2\n",
      "Number of gt: 7\n",
      "Average precision 0.0\n",
      "\n",
      "mAP as the average of AP across the 2 categories is 0.3729\n",
      "Precision at 0.8 recall: (0.0, 0.9990000128746033)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGxtJREFUeJzt3XuUXnV97/H3JzOTBAhJwEm4JCEJlwARQTRyqT0CB+QkWR7SdrEQKiqWSuWA6NFqabVKqa5z0GN7WsHanANekDssOVNIpUoRtBKbQe7B0BiBDOGS24SQZGaSyff88dvPniczk5k9ZPY8c/m81noW+/J79vPdk2F/Zu/fs39bEYGZmRnAuFoXYGZmw4dDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FG3UkvSDpnL2sO1NSyxDUcImkn5f8GXMkhaT6Mj/HxhaHgpmZ5RwKZmaWcyjYaPUeSSslbZb0HUkTq1dK+gtJG7JLTR+qWj5F0vclrZf0oqQvShqXrfsHSXdXtb1O0oOS1F8xko6T9GNJmyStknRBtvw0Sa9Kqqtq+/uSnsqmx0m6WtJvJG2UdKekg/f9x2PWO4eCjVYfAv4LcBQwD/hi1bpDgUZgBvBRYKmkY7N13wSmAEcCZwAfAT6WrfsscGLWX/CfgEuBj0Y/Y8VIOgD4MXArMB24CPiWpLdHxHJgG/Cfq97yh1lbgKuA38tqORzYDNxQ/MdgNjAOBRutro+ItRGxCfgq6UBc7S8joj0iHgbuBy7I/lr/IPDnEbE1Il4AvgF8GCAitgMXA38D/AD4ZEQU6bT+APBCRHwnInZFxK+Ae4Dzs/W3VeqTdCCwOFsG8CfAFyKiJSLagWuA8925bGXxL5aNVmurpl8k/ZVdsTkitvWyvhEYn81Xr5tRmYmIf5e0hvQX/50Fa5kNnCqptWpZPXBzNn0r8AtJlwN/APwqIl6seu8PJe2uem8ncEjBzzYbEJ8p2Gg1q2r6CGBd1fxB2SWd7us3ADtJB+LqdS9XZiRdAUzI2n++YC1rgYcjYmrVa1JEXA4QEStJ4bOIPS8dVd67qNt7J0bEyz0+xWwQOBRstLpC0sysU/YvgDu6rf8rSeOzvoEPAHdFRCfpr/+vSjpQ0mzgM6RLRUiaB3yFdAnpw8DnJb2zQC33AfMkfVhSQ/Z6j6Tjq9rcSuo/eB9wV9Xyb2f1zM5qmCZpyYB+EmYD4FCw0epW4F+ANdnrK1XrXiV12K4DbgE+ERG/ztZ9ktTxuwb4ebadm7Jr+D8ArouIJyPiP0hhc7OkCX0VEhFbgXOBC7PPfBW4jnTGUXEbcCbwrxGxoWr53wFNwL9I2gosB04t/mMwGxj5ITtmZlbhMwUzM8s5FMzMLOdQMDOznEPBzMxyI+7mtcbGxpgzZ06tyzAzG1Eee+yxDRExrb92Iy4U5syZQ3Nzc63LMDMbUSS92H8rXz4yM7MqDgUzM8s5FMzMLOdQMDOznEPBzMxyDgUzM8uVFgqSbpL0uqRn9rJekv5e0mpJT0l6V1m1mJlZMWXep/Bd4Hrg+3tZvwg4JnudCvwDBYcE3r27/zbDxTifi5nZCFJaKETEI5Lm9NFkCfD97KHnyyVNlXRYRLzS13bffBN+9rNBLHQITJsGh2QPT4xIr44O2LkTDjssze/eDXV1MKHPkfnNzMpVyzuaZ7Dnc3RbsmU9QkHSZcBlANOmzeGll9IBdLjbvh22bIGpU6G+visQKo+wkKChYc/3TJ+e1u/aBZ2d6b2HHJKmOztTeHR2dq2vTO/enYLmwAOhsbGr3bZt6XOmTev5+b1NVwIqomdtFbt2wY4dMHly2sfOTjj88PQ5lf3avj2dJY0fD/vtB1OmdK03s+GrlqHQ2yGi1yf+RMRSYCnAccctiKOOGjl/UXd0wMsvw9at6aAodV1S2rGj6wDc0QHt7dDW1nXw3Lw5HVCff37PbUpdB/NqW7bAxIl7/my2bUsBOnFi/++vDgYpHdC7BwmkUOjoSKFRaVtXl6br6rq2X5mubKsSVu3taTszZqT9i9gz8KqnGxrS/lTCqrMzTdfXpzoADj64589sypQUTBFpfWW/Kz+TKVNg0qSR8ceF2VCqZSi0sOfD1Wey58PVR4Xx42Hu3Lf+/spBd1/a79yZDuKVg3TlQDhuXNdBu7ftdHTs2ab6FdF1FrJrF7zxRppua0vvg3Twb29PB+eGhnTpb/fu1H77dnj11Z6fXdl25VWptfsZzbhxaRtSCrxKKFXXXHl/9zOe6sB629vgoINSm2nTun5ekyen6c2b05malEK8EiTjx7u/yEanWoZCE3ClpNtJHcxb+utPGIsGesmlt/YNDXu/FNTXdvo6G5PSAbJiypSBbR8GHni92b07vdrauupqb09nFBJs2tR1ZlLR3p4O8OPGpTB7/vmeB/n67P+MurquAOmuoSGd/ey/f3o/pDA59NCusDzggK669rYds+GktFCQVHkQeaOkFuDLQANARHwbWAYsBlYD24GPlVWLDU+DcYAcNy69qgOqciCGdNAuYtu2FBRSOtt4443030rfDaQwaWhIAbRzZ/rc117rupxVX79nuFTOWOrqui6FHXFECpGjjkp1OiRsuCnz20cX9bM+gCvK+nyzgTjggD3D5LDDir83IoVEa2vqP6r0ibS1pbCqXNbauhXWr0/B8cQTKSwaG1OfR11d6qw/6KCu7TY0uM/Dht6Ie56C2XBT6UifPj29+rJrF6xdC+vWpbOT9evTGcaECfDMMz1DQEqhMW1aCpijj+66tGVWBv96mQ2h+vr0xYPqLx/s3g2rV6f+j4aGdNmpvT0Fwptvwuuvw29+ky47NTenzvG5c7u+XdXQAEcemQJlv/1qt282OjgUzGps3DiYN6/vNm++CU8/nfowWluhpaXrm1YTJsBTT3W1nTs3Xf7auROOPbbc2m30cSiYjQCTJsHpp3fN796dDvrbtqWziI4O2LgxnTVs3tz1Nd0VK2Dx4q57Ncz641AwG4EqZwgTJvQ84O/cmfosnn02fVW4qSl1YM+bl+YnT97z21pm1RwKZqNMQwPMnp1era2wfHk6e1i3LvU51Nen/55yCsya1f/2bGxxKJiNYlOnwsKFqUP69dfTN59aWtIYWa2t6VtThx8OJ56YziIqN+HZ2OVQMBsDpDRcR2W03vZ2+MUvUt/Epk3w4ovpDOPgg+HMM/e8Z8PGFoeC2Rg0YQKcdVaabmuDJ5+EV15JN9jdc08Kh2OPTXdgj5TBJ21wOBTMxriJE+HU7PFWzz2XLjFt3JhCYsIEeMc70ss3zY0N/mc2s9zxx6dXZyc8+mgKh7a29E2mhQuLjyVlI5cH/zWzHurq4Hd/F845J/U5vPwy3H8//OQnta7MyuYzBTPbKwne//4UDCtWpLOGbdvSDXEDHY7dRgafKZhZvw4+GM4+O32ttaUFbrsNfvWrWldlZXAomFkh9fXpDGHChPTUvOZm+MEPup43YaODQ8HMBuTkk+GMM9LgfJs2pbMGGz0cCmY2YPX1sGhR6nN47TW4446u52jbyOZQMLO3RILf+Z30XIeNG+GnP611RTYYHApmtk/OPTeFwqpV6YzBRjaHgpntk8qlpPXr02isL7xQ64psXzgUzGyfSanzedMmeOih9NAfG5kcCmY2KA44AObPT/cy/OhHta7G3iqHgpkNmpkz01nCa6+lfgYbeRwKZjZopPQs6Y0b4b770vMabGRxKJjZoDrooPQwn/Xr4d57a12NDZRDwcwG3fz5sGNH6njeurXW1dhAOBTMbNDV16cb29avhx/+sNbV2EA4FMysFFOnwuTJsGED3HJLrauxohwKZlaa005Lz1/YtCk9+9njIw1/DgUzK9WiRdDamu5fuOWWNFaSDV8OBTMrlZSew7BhQwqGe+6BN96odVW2Nw4FMxsSixfDoYemJ7fdf3+tq7G9KTUUJC2UtErSaklX97L+CEkPSXpc0lOSFpdZj5nV1rHHwvjx6eY23/E8PJUWCpLqgBuARcB84CJJ87s1+yJwZ0ScDFwIfKusesxseHj729O9C489VutKrDdlnimcAqyOiDUR0QHcDizp1iaAydn0FGBdifWY2TDQ2Ai7dsHatdDeXutqrLsyQ2EGsLZqviVbVu0a4GJJLcAy4JO9bUjSZZKaJTW3tq4vo1YzGyISLFiQLh/dc0+tq7HuygwF9bKs+7eULwK+GxEzgcXAzZJ61BQRSyNiQUQsmDp1WgmlmtlQmj493di2ZUutK7HuygyFFmBW1fxMel4euhS4EyAiHgUmAo0l1mRmw8T06Wl8pOXLa12JVSszFFYAx0iaK2k8qSO5qVubl4CzASQdTwoFXx8yGwMqfQuTJtW6EqtWWihExC7gSuAB4DnSt4yelXStpPOyZp8FPi7pSeA24JII3whvNhYccACcdBJM8xXhYaW+zI1HxDJSB3L1si9VTa8E3ltmDWZmVpzvaDYzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPL1RdtKGkGMLv6PRHxSBlFmZlZbRQKBUnXAR8EVgKd2eIA+gwFSQuBvwPqgP8bEf+zlzYXANdk23syIv6waPFmZja4ip4p/B5wbES0F92wpDrgBuD9QAuwQlJTRKysanMM8OfAeyNis6TpxUs3M7PBVrRPYQ3QMMBtnwKsjog1EdEB3A4s6dbm48ANEbEZICJeH+BnmJnZICp6prAdeELSg0B+thARV/XxnhnA2qr5FuDUbm3mAUj6N9Ilpmsi4kcFazIzs0FWNBSastdAqJdl0cvnHwOcCcwEfibphIho3WND0mXAZQCHHHLEAMswM7OiCoVCRHxP0niyv+yBVRGxs5+3tQCzquZnAut6abM829ZvJa0ihcSKbp+/FFgKcNxxC7oHi5mZDZJCfQqSzgT+g9Rx/C3geUnv6+dtK4BjJM3NAuVCep5t3AuclX1GIyl01hSu3szMBlXRy0ffAM6NiFUAkuYBtwHv3tsbImKXpCuBB0j9BTdFxLOSrgWaI6IpW3eupMpXXT8XERvf+u6Ymdm+KBoKDZVAAIiI5yX1+22kiFgGLOu27EtV0wF8JnuZmVmNFQ2FZkk3Ajdn8x8CHiunJDMzq5WioXA5cAVwFelbRY+Q+hbMzGwUKfrto3bgb7KXmZmNUn2GgqQ7I+ICSU/T8x4DIuLE0iozM7Mh19+Zwqey/36g7ELMzKz2+rxPISJeySY3AGsj4kVgAnASPW9EMzOzEa7ogHiPABOzZyo8CHwM+G5ZRZmZWW0UDQVFxHbgD4BvRsTvA/PLK8vMzGqhcChIOp10f8L92bLCT20zM7ORoWgofJr0MJwfZkNVHAk8VF5ZZmZWC0XvU3gYeLhqfg3pRjYzMxtF+rtP4X9HxKcl/RO936dwXmmVmZnZkOvvTKEy1tH/KrsQMzOrvT5DISIqg941AzsiYjeApDrS/QpmZjaKFO1ofhDYv2p+P+Ang1+OmZnVUtFQmBgRb1Zmsun9+2hvZmYjUNFQ2CbpXZUZSe8GdpRTkpmZ1UrRG9A+DdwlqTLe0WHAB8spyczMaqXofQorJB0HHEt6yM6vI2JnqZWZmdmQK3T5SNL+wJ8Bn4qIp4E5kjyctpnZKFO0T+E7QAdwejbfAnyllIrMzKxmiobCURHxNWAnQETsIF1GMjOzUaRoKHRI2o9sqAtJRwHtpVVlZmY1UfTbR18GfgTMknQL8F7gkrKKMjOz2ug3FCQJ+DXpATunkS4bfSoiNpRcm5mZDbF+QyEiQtK9EfFuuh6wY2Zmo1DRPoXlkt5TaiVmZlZzRfsUzgI+IekFYBvpElJExIllFWZmZkOvaCgsKrUKMzMbFvp78tpE4BPA0cDTwI0RsWsoCjMzs6HXX5/C94AFpEBYBHyj9IrMzKxm+rt8ND8i3gEg6Ubg38svyczMaqW/M4V8JFRfNjIzG/36C4WTJL2RvbYCJ1amJb3R38YlLZS0StJqSVf30e58SSFpwUB3wMzMBk+fl48iou6tblhSHXAD8H7SqKorJDVFxMpu7Q4ErgJ++VY/y8zMBkfRm9feilOA1RGxJiI6gNuBJb20+2vga0BbibWYmVkBZYbCDGBt1XxLtiwn6WRgVkTc19eGJF0mqVlSc2vr+sGv1MzMgHJDobfnLUS+UhoH/C3w2f42FBFLI2JBRCyYOnXaIJZoZmbVygyFFmBW1fxMYF3V/IHACcBPs+EzTgOa3NlsZlY7ZYbCCuAYSXMljQcuBJoqKyNiS0Q0RsSciJgDLAfOi4jmEmsyM7M+lBYK2X0NVwIPAM8Bd0bEs5KulXReWZ9rZmZvXdEB8d6SiFgGLOu27Et7aXtmmbWYmVn/yrx8ZGZmI4xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCznUDAzs5xDwczMcg4FMzPLORTMzCxXaihIWihplaTVkq7uZf1nJK2U9JSkByXNLrMeMzPrW2mhIKkOuAFYBMwHLpI0v1uzx4EFEXEicDfwtbLqMTOz/pV5pnAKsDoi1kREB3A7sKS6QUQ8FBHbs9nlwMwS6zEzs36UGQozgLVV8y3Zsr25FPjn3lZIukxSs6Tm1tb1g1iimZlVKzMU1Muy6LWhdDGwAPh6b+sjYmlELIiIBVOnThvEEs3MrFp9idtuAWZVzc8E1nVvJOkc4AvAGRHRXmI9ZmbWjzLPFFYAx0iaK2k8cCHQVN1A0snAPwLnRcTrJdZiZmYFlBYKEbELuBJ4AHgOuDMinpV0raTzsmZfByYBd0l6QlLTXjZnZmZDoMzLR0TEMmBZt2Vfqpo+p8zPNzOzgfEdzWZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlnMomJlZzqFgZmY5h4KZmeUcCmZmlis1FCQtlLRK0mpJV/eyfoKkO7L1v5Q0p8x6zMysb6WFgqQ64AZgETAfuEjS/G7NLgU2R8TRwN8C15VVj5mZ9a++xG2fAqyOiDUAkm4HlgArq9osAa7Jpu8GrpekiIi+NtzWBn23MLORoK2t1hVYd2WGwgxgbdV8C3Dq3tpExC5JW4C3ARuqG0m6DLgsm+04++zJa8ZuLOw8CBo217qK2hnL+z8a912C7W3Q2VmgcSPdjg1jzL7u/+wijcoMBfWyrPuBvEgbImIpsBRAUnPEGwv2vbyRKe1/m/d/DBrL+w6V/Q/vf8nK7GhuAWZVzc8E1u2tjaR6YAqwqcSazMysD2WGwgrgGElzJY0HLgSaurVpAj6aTZ8P/Gt//QlmZlae0i4fZX0EVwIPAHXATRHxrKRrgeaIaAJuBG6WtJp0hnBhgU0vLavmEcL7P3aN5X0H7/+Q7L/8h7mZmVX4jmYzM8s5FMzMLDdsQ2GsD5FRYP8/I2mlpKckPSip0HeQR4L+9r2q3fmSQtKo+ppikf2XdEH27/+spFuHusYyFfjdP0LSQ5Iez37/F9eizjJIuknS65Ke2ct6Sfr77GfzlKR3DXoRETHsXqSO6d8ARwLjgSeB+d3a/Dfg29n0hcAdta57iPf/LGD/bPry0bL/RfY9a3cg8AiwHFhQ67qH+N/+GOBx4KBsfnqt6x7i/V8KXJ5NzwdeqHXdg7j/7wPeBTyzl/WLgX8m3eN1GvDLwa5huJ4p5ENkREQHUBkio9oS4HvZ9N3A2ZJ6uxluJOp3/yPioYjYns0uJ90HMhoU+bcH+Gvga8BoGyihyP5/HLghIjYDRMTrQ1xjmYrsfwCTs+kp9Lz/acSKiEfo+16tJcD3I1kOTJV02GDWMFxDobchMmbsrU1E7AIqQ2SMBkX2v9qlpL8eRoN+913SycCsiLhvKAsbIkX+7ecB8yT9m6TlkhYOWXXlK7L/1wAXS2oBlgGfHJrShoWBHhsGrMxhLvbFoA2RMUIV3jdJFwMLgDNKrWjo9LnvksaRRtS9ZKgKGmJF/u3rSZeQziSdIf5M0gkR0VpybUOhyP5fBHw3Ir4h6XTSvU4nRMTu8surudKPe8P1TGGsD5FRZP+RdA7wBeC8iGgfotrK1t++HwicAPxU0guk66pNo6izuejv/v+LiJ0R8VtgFSkkRoMi+38pcCdARDwKTCQNFjcWFDo27IvhGgpjfYiMfvc/u4Tyj6RAGE3XlPvc94jYEhGNETEnIuaQ+lPOi4jm2pQ76Ir87t9L+qIBkhpJl5PWDGmV5Smy/y8BZwNIOp4UCuuHtMraaQI+kn0L6TRgS0S8MpgfMCwvH0V5Q2SMCAX3/+vAJOCurH/9pYg4r2ZFD5KC+z5qFdz/B4BzJa0EOoHPRcTG2lU9eAru/2eB/yPpv5MunVwyWv4glHQb6bJgY9Zn8mWgASAivk3qQ1kMrAa2Ax8b9BpGyc/SzMwGwXC9fGRmZjXgUDAzs5xDwczMcg4FMzPLORTMzCznUDDrRlKnpCckPSPpnyRNHeTtXyLp+mz6Gkl/OpjbN9sXDgWznnZExDsj4gTSPTBX1Logs6HiUDDr26NUDTgm6XOSVmRj2f9V1fKPZMuelHRztuy/Zs/6eFzSTyQdUoP6zQZkWN7RbDYcSKojDadwYzZ/LmmMoVNIA5M1SXofsJE0BtV7I2KDpIOzTfwcOC0iQtIfA58n3Y1rNmw5FMx62k/SE8Ac4DHgx9nyc7PX49n8JFJInATcHREbACKiMjDjTOCObLz78cBvh6R6s33gy0dmPe2IiHcCs0kH80qfgoD/kfU3vDMijo6IG7PlvY0X803g+oh4B/AnpIHbzIY1h4LZXkTEFuAq4E8lNZAGafsjSZMAJM2QNB14ELhA0tuy5ZXLR1OAl7Ppj2I2AvjykVkfIuJxSU8CF0bEzdlQzY9mI9O+CVycjeL5VeBhSZ2ky0uXkJ4Qdpekl0lDfM+txT6YDYRHSTUzs5wvH5mZWc6hYGZmOYeCmZnlHApmZpZzKJiZWc6hYGZmOYeCmZnl/j9oPUgrhWO39AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "file_prefix = 'SER/'\n",
    "\n",
    "results_path = '/home/marmot/mnt/ai4edevshare/api_outputs/Benchmark/190904/5618_detections_train_cct_20_val_ss1_20190908093215.json'\n",
    "\n",
    "gt_db_path = '/beaver_disk/camtrap/ss_season1/benchmark/SnapshotSerengetiBboxesS01_20190903_val.json'\n",
    "\n",
    "detection_res = make_detection_res(results_path, file_prefix=file_prefix)\n",
    "\n",
    "gt_indexed = get_gt_db(gt_db_path)\n",
    "\n",
    "per_image_gts, per_image_detections = detector_eval.get_per_image_gts_and_detections(gt_indexed, detection_res)\n",
    "\n",
    "per_cat_metrics = detector_eval.compute_precision_recall_bbox(per_image_detections, per_image_gts, num_gt_classes, \n",
    "                                                         matching_iou_threshold=0.5)\n",
    "\n",
    "show_metrics(per_cat_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## empty vs non-empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9232630487383339\n"
     ]
    }
   ],
   "source": [
    "accuracy = detector_eval.compute_emptiness_accuracy(gt_indexed, detection_res, threshold=empty_threshold)\n",
    "\n",
    "print('Accuracy:', accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
